# -*- coding:utf8 -*-

''' 第一高考专业解析 '''
import importlib
import re, sys, json, datetime, random, time
from scrapy.selector import Selector
from scrapy.exceptions import CloseSpider
try:
    from scrapy.spiders import Spider
except:
    from scrapy.spider import BaseSpider as Spider

from scrapy.http import Request, FormRequest
from scrapy.utils.response import get_base_url
from scrapy.utils.url import urljoin_rfc

from gaokaopai.items import *
from gaokaopai.dao import *
from gaokaopai.util import *

importlib.reload(sys)
# sys.setdefaultencoding("utf-8")

class BaoKaoDaXueUniversitySpider(Spider):
    name        = 'baokaodaxue_university'
    allow       = ['baokaodaxue.com']

    def __init__(self, *args, **kwargs):
        super(BaoKaoDaXueUniversitySpider, self).__init__(*args, **kwargs)

    def start_requests(self):
        yield Request("http://www.baokaodaxue.com/bkdx/search/college?page=1", callback=self.parse_list, dont_filter=True, meta={'page':1})

    def parse_list(self, response):
        meta = response.meta
        base_url = get_base_url(response)

        for item_dom in response.xpath(u"//div[contains(@class, 'result-college-item')]/.//div[@class='college-name']/a"):
            name = ''.join(item_dom.xpath(u"./text()").extract()).strip()
            url = ''.join(item_dom.xpath(u"./@href").extract()).strip()

            qs = getBKDXCid(url)
            cid = ''.join(qs['cid'])

            university = BKUniversity()
            university['table'] = 't_bk_university'
            university['name']  = name
            university['cid']   = cid

            yield university


        '''分页效果'''
        next_page = int(meta['page'])
        if next_page <= 133:
            yield Request("http://www.baokaodaxue.com/bkdx/search/college?page=%s"%(next_page+1), callback=self.parse_list, meta={'page':int(next_page+1)}, dont_filter=True)




